Module Imports¶

In [1]:
import numpy as np 
import pandas as pd
import seaborn as sns
import itertools
import os 
import matplotlib.pyplot as plt


import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.layers import Dense, Activation
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.metrics import categorical_crossentropy
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.preprocessing import image
from tensorflow.keras.models import Model
from tensorflow.keras.applications import imagenet_utils

from sklearn.metrics import confusion_matrix, precision_score, recall_score, accuracy_score, f1_score

Import of MobileNet¶

In [2]:
mobile = tf.keras.applications.mobilenet.MobileNet()

Metal device set to: Apple M2

2023-03-17 12:02:54.504952: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:305] Could not identify NUMA node of platform GPU ID 0, defaulting to 0. Your kernel may not have been built with NUMA support.
2023-03-17 12:02:54.505041: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:271] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 0 MB memory) -> physical PluggableDevice (device: 0, name: METAL, pci bus id: <undefined>)

Data import and formatting¶

In [3]:
train_dir = "/Users/preslav/Downloads/cw_cop528/imageset/train"
test_dir = "/Users/preslav/Downloads/cw_cop528/imageset/val"
In [4]:
# import data, whilist preprocessing pixel values in the range [-1, 1]
train_batches = ImageDataGenerator(preprocessing_function=tf.keras.applications.mobilenet.preprocess_input, validation_split=0.2).flow_from_directory(
directory=train_dir, target_size = (224, 224), subset = "training" ,batch_size = 32, seed = 2)

validation_batches = ImageDataGenerator(preprocessing_function=tf.keras.applications.mobilenet.preprocess_input, validation_split=0.2).flow_from_directory(
directory=train_dir, target_size = (224, 224), subset = "validation" ,batch_size = 32, seed = 2)

test_batches = ImageDataGenerator(preprocessing_function=tf.keras.applications.mobilenet.preprocess_input).flow_from_directory(
directory=test_dir, target_size = (224, 224),batch_size = 32, seed = 2, shuffle = False)

Found 7578 images belonging to 10 classes.
Found 1891 images belonging to 10 classes.
Found 3925 images belonging to 10 classes.
In [5]:
# import of the class labels names and their total number 
class_names = list(train_batches.class_indices.keys())
num_classes = len(class_names)
print(class_names)
print(num_classes)

['building', 'dog', 'fish', 'gas_station', 'golf', 'musician', 'parachute', 'radio', 'saw', 'vehicle']
10

Examining how MobileNet works¶

In [6]:
# pixel values are brough from standartization to normalization,
# so that they can be propertly displayed
img, lbl = next(train_batches)
img_disp = (img + 1)/2
In [7]:
# display image
plt.imshow(img_disp[24])
plt.tight_layout()
plt.show()

# get predictions
predictions_new = mobile.predict(img_disp)
results_new = imagenet_utils.decode_predictions(predictions_new)

# display the highest 5 predictions in a dataframe
mobile_net_predictions = pd.DataFrame(results_new[24])
mobile_net_predictions.columns = ["actual_label", "preedicted_label", "propability"]
display(mobile_net_predictions)

2023-03-17 12:02:55.467408: W tensorflow/core/platform/profile_utils/cpu_utils.cc:128] Failed to get CPU frequency: 0 Hz
2023-03-17 12:02:55.592597: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.

1/1 [==============================] - 0s 366ms/step
actual_label preedicted_label propability
0 n04208210 shovel 0.161206
1 n03000684 chain_saw 0.114571
2 n04461696 tow_truck 0.066798
3 n03594945 jeep 0.064021
4 n02493793 spider_monkey 0.061292

Transfer Learning, with fixed parameters¶

In [8]:
# this part of the code has been inspired by the following tutorial: 
# https://www.youtube.com/watch?v=Zrt76AIbeh4
In [9]:
# freeze all the layers, so they are not trainable and no weights nor biases are be updated
for layer in mobile.layers:
    layer.trainable = False
In [10]:
# replace last 5 layers with our own 2 dense layers
x_remove_layers = mobile.layers[-5].output
output_non_trainable = Dense(units = 10, activation = "softmax")(x_remove_layers)
output_non_trainable = tf.keras.layers.Reshape(target_shape=(10,))(output_non_trainable)
In [11]:
# create module, that takes MobileNet and transforms it with our desired final layers,
# in place of the MobileNet's last 5 layers
model_non_trainable = Model(inputs = mobile.input, outputs = output_non_trainable)
In [12]:
model_non_trainable.compile(optimizer = Adam(learning_rate = 0.0001), 
                            loss = "categorical_crossentropy", 
                            metrics = ["accuracy"])
In [13]:
# it can be noticed, that there are onlt 10250 trainable parameters,
# which come only from the layers added by us 
model_non_trainable.summary()

Model: "model"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_1 (InputLayer)        [(None, 224, 224, 3)]     0         
                                                                 
 conv1 (Conv2D)              (None, 112, 112, 32)      864       
                                                                 
 conv1_bn (BatchNormalizatio  (None, 112, 112, 32)     128       
 n)                                                              
                                                                 
 conv1_relu (ReLU)           (None, 112, 112, 32)      0         
                                                                 
 conv_dw_1 (DepthwiseConv2D)  (None, 112, 112, 32)     288       
                                                                 
 conv_dw_1_bn (BatchNormaliz  (None, 112, 112, 32)     128       
 ation)                                                          
                                                                 
 conv_dw_1_relu (ReLU)       (None, 112, 112, 32)      0         
                                                                 
 conv_pw_1 (Conv2D)          (None, 112, 112, 64)      2048      
                                                                 
 conv_pw_1_bn (BatchNormaliz  (None, 112, 112, 64)     256       
 ation)                                                          
                                                                 
 conv_pw_1_relu (ReLU)       (None, 112, 112, 64)      0         
                                                                 
 conv_pad_2 (ZeroPadding2D)  (None, 113, 113, 64)      0         
                                                                 
 conv_dw_2 (DepthwiseConv2D)  (None, 56, 56, 64)       576       
                                                                 
 conv_dw_2_bn (BatchNormaliz  (None, 56, 56, 64)       256       
 ation)                                                          
                                                                 
 conv_dw_2_relu (ReLU)       (None, 56, 56, 64)        0         
                                                                 
 conv_pw_2 (Conv2D)          (None, 56, 56, 128)       8192      
                                                                 
 conv_pw_2_bn (BatchNormaliz  (None, 56, 56, 128)      512       
 ation)                                                          
                                                                 
 conv_pw_2_relu (ReLU)       (None, 56, 56, 128)       0         
                                                                 
 conv_dw_3 (DepthwiseConv2D)  (None, 56, 56, 128)      1152      
                                                                 
 conv_dw_3_bn (BatchNormaliz  (None, 56, 56, 128)      512       
 ation)                                                          
                                                                 
 conv_dw_3_relu (ReLU)       (None, 56, 56, 128)       0         
                                                                 
 conv_pw_3 (Conv2D)          (None, 56, 56, 128)       16384     
                                                                 
 conv_pw_3_bn (BatchNormaliz  (None, 56, 56, 128)      512       
 ation)                                                          
                                                                 
 conv_pw_3_relu (ReLU)       (None, 56, 56, 128)       0         
                                                                 
 conv_pad_4 (ZeroPadding2D)  (None, 57, 57, 128)       0         
                                                                 
 conv_dw_4 (DepthwiseConv2D)  (None, 28, 28, 128)      1152      
                                                                 
 conv_dw_4_bn (BatchNormaliz  (None, 28, 28, 128)      512       
 ation)                                                          
                                                                 
 conv_dw_4_relu (ReLU)       (None, 28, 28, 128)       0         
                                                                 
 conv_pw_4 (Conv2D)          (None, 28, 28, 256)       32768     
                                                                 
 conv_pw_4_bn (BatchNormaliz  (None, 28, 28, 256)      1024      
 ation)                                                          
                                                                 
 conv_pw_4_relu (ReLU)       (None, 28, 28, 256)       0         
                                                                 
 conv_dw_5 (DepthwiseConv2D)  (None, 28, 28, 256)      2304      
                                                                 
 conv_dw_5_bn (BatchNormaliz  (None, 28, 28, 256)      1024      
 ation)                                                          
                                                                 
 conv_dw_5_relu (ReLU)       (None, 28, 28, 256)       0         
                                                                 
 conv_pw_5 (Conv2D)          (None, 28, 28, 256)       65536     
                                                                 
 conv_pw_5_bn (BatchNormaliz  (None, 28, 28, 256)      1024      
 ation)                                                          
                                                                 
 conv_pw_5_relu (ReLU)       (None, 28, 28, 256)       0         
                                                                 
 conv_pad_6 (ZeroPadding2D)  (None, 29, 29, 256)       0         
                                                                 
 conv_dw_6 (DepthwiseConv2D)  (None, 14, 14, 256)      2304      
                                                                 
 conv_dw_6_bn (BatchNormaliz  (None, 14, 14, 256)      1024      
 ation)                                                          
                                                                 
 conv_dw_6_relu (ReLU)       (None, 14, 14, 256)       0         
                                                                 
 conv_pw_6 (Conv2D)          (None, 14, 14, 512)       131072    
                                                                 
 conv_pw_6_bn (BatchNormaliz  (None, 14, 14, 512)      2048      
 ation)                                                          
                                                                 
 conv_pw_6_relu (ReLU)       (None, 14, 14, 512)       0         
                                                                 
 conv_dw_7 (DepthwiseConv2D)  (None, 14, 14, 512)      4608      
                                                                 
 conv_dw_7_bn (BatchNormaliz  (None, 14, 14, 512)      2048      
 ation)                                                          
                                                                 
 conv_dw_7_relu (ReLU)       (None, 14, 14, 512)       0         
                                                                 
 conv_pw_7 (Conv2D)          (None, 14, 14, 512)       262144    
                                                                 
 conv_pw_7_bn (BatchNormaliz  (None, 14, 14, 512)      2048      
 ation)                                                          
                                                                 
 conv_pw_7_relu (ReLU)       (None, 14, 14, 512)       0         
                                                                 
 conv_dw_8 (DepthwiseConv2D)  (None, 14, 14, 512)      4608      
                                                                 
 conv_dw_8_bn (BatchNormaliz  (None, 14, 14, 512)      2048      
 ation)                                                          
                                                                 
 conv_dw_8_relu (ReLU)       (None, 14, 14, 512)       0         
                                                                 
 conv_pw_8 (Conv2D)          (None, 14, 14, 512)       262144    
                                                                 
 conv_pw_8_bn (BatchNormaliz  (None, 14, 14, 512)      2048      
 ation)                                                          
                                                                 
 conv_pw_8_relu (ReLU)       (None, 14, 14, 512)       0         
                                                                 
 conv_dw_9 (DepthwiseConv2D)  (None, 14, 14, 512)      4608      
                                                                 
 conv_dw_9_bn (BatchNormaliz  (None, 14, 14, 512)      2048      
 ation)                                                          
                                                                 
 conv_dw_9_relu (ReLU)       (None, 14, 14, 512)       0         
                                                                 
 conv_pw_9 (Conv2D)          (None, 14, 14, 512)       262144    
                                                                 
 conv_pw_9_bn (BatchNormaliz  (None, 14, 14, 512)      2048      
 ation)                                                          
                                                                 
 conv_pw_9_relu (ReLU)       (None, 14, 14, 512)       0         
                                                                 
 conv_dw_10 (DepthwiseConv2D  (None, 14, 14, 512)      4608      
 )                                                               
                                                                 
 conv_dw_10_bn (BatchNormali  (None, 14, 14, 512)      2048      
 zation)                                                         
                                                                 
 conv_dw_10_relu (ReLU)      (None, 14, 14, 512)       0         
                                                                 
 conv_pw_10 (Conv2D)         (None, 14, 14, 512)       262144    
                                                                 
 conv_pw_10_bn (BatchNormali  (None, 14, 14, 512)      2048      
 zation)                                                         
                                                                 
 conv_pw_10_relu (ReLU)      (None, 14, 14, 512)       0         
                                                                 
 conv_dw_11 (DepthwiseConv2D  (None, 14, 14, 512)      4608      
 )                                                               
                                                                 
 conv_dw_11_bn (BatchNormali  (None, 14, 14, 512)      2048      
 zation)                                                         
                                                                 
 conv_dw_11_relu (ReLU)      (None, 14, 14, 512)       0         
                                                                 
 conv_pw_11 (Conv2D)         (None, 14, 14, 512)       262144    
                                                                 
 conv_pw_11_bn (BatchNormali  (None, 14, 14, 512)      2048      
 zation)                                                         
                                                                 
 conv_pw_11_relu (ReLU)      (None, 14, 14, 512)       0         
                                                                 
 conv_pad_12 (ZeroPadding2D)  (None, 15, 15, 512)      0         
                                                                 
 conv_dw_12 (DepthwiseConv2D  (None, 7, 7, 512)        4608      
 )                                                               
                                                                 
 conv_dw_12_bn (BatchNormali  (None, 7, 7, 512)        2048      
 zation)                                                         
                                                                 
 conv_dw_12_relu (ReLU)      (None, 7, 7, 512)         0         
                                                                 
 conv_pw_12 (Conv2D)         (None, 7, 7, 1024)        524288    
                                                                 
 conv_pw_12_bn (BatchNormali  (None, 7, 7, 1024)       4096      
 zation)                                                         
                                                                 
 conv_pw_12_relu (ReLU)      (None, 7, 7, 1024)        0         
                                                                 
 conv_dw_13 (DepthwiseConv2D  (None, 7, 7, 1024)       9216      
 )                                                               
                                                                 
 conv_dw_13_bn (BatchNormali  (None, 7, 7, 1024)       4096      
 zation)                                                         
                                                                 
 conv_dw_13_relu (ReLU)      (None, 7, 7, 1024)        0         
                                                                 
 conv_pw_13 (Conv2D)         (None, 7, 7, 1024)        1048576   
                                                                 
 conv_pw_13_bn (BatchNormali  (None, 7, 7, 1024)       4096      
 zation)                                                         
                                                                 
 conv_pw_13_relu (ReLU)      (None, 7, 7, 1024)        0         
                                                                 
 global_average_pooling2d (G  (None, 1, 1, 1024)       0         
 lobalAveragePooling2D)                                          
                                                                 
 dense (Dense)               (None, 1, 1, 10)          10250     
                                                                 
 reshape (Reshape)           (None, 10)                0         
                                                                 
=================================================================
Total params: 3,239,114
Trainable params: 10,250
Non-trainable params: 3,228,864
_________________________________________________________________
In [14]:
epochs = 20
history_non_trainable = model_non_trainable.fit(x = train_batches, 
                    validation_data = validation_batches, 
                    epochs = epochs)

Epoch 1/20

2023-03-17 12:02:56.755772: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.

237/237 [==============================] - ETA: 0s - loss: 1.6925 - accuracy: 0.4677
2023-03-17 12:03:15.694863: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.

237/237 [==============================] - 24s 99ms/step - loss: 1.6925 - accuracy: 0.4677 - val_loss: 0.8438 - val_accuracy: 0.8006
Epoch 2/20
237/237 [==============================] - 23s 96ms/step - loss: 0.5574 - accuracy: 0.8853 - val_loss: 0.4161 - val_accuracy: 0.9138
Epoch 3/20
237/237 [==============================] - 23s 96ms/step - loss: 0.3160 - accuracy: 0.9390 - val_loss: 0.2803 - val_accuracy: 0.9418
Epoch 4/20
237/237 [==============================] - 23s 97ms/step - loss: 0.2241 - accuracy: 0.9543 - val_loss: 0.2171 - val_accuracy: 0.9513
Epoch 5/20
237/237 [==============================] - 23s 97ms/step - loss: 0.1746 - accuracy: 0.9613 - val_loss: 0.1782 - val_accuracy: 0.9582
Epoch 6/20
237/237 [==============================] - 23s 96ms/step - loss: 0.1433 - accuracy: 0.9679 - val_loss: 0.1542 - val_accuracy: 0.9651
Epoch 7/20
237/237 [==============================] - 23s 96ms/step - loss: 0.1210 - accuracy: 0.9741 - val_loss: 0.1340 - val_accuracy: 0.9699
Epoch 8/20
237/237 [==============================] - 23s 96ms/step - loss: 0.1046 - accuracy: 0.9782 - val_loss: 0.1206 - val_accuracy: 0.9720
Epoch 9/20
237/237 [==============================] - 23s 96ms/step - loss: 0.0916 - accuracy: 0.9809 - val_loss: 0.1091 - val_accuracy: 0.9757
Epoch 10/20
237/237 [==============================] - 23s 96ms/step - loss: 0.0810 - accuracy: 0.9838 - val_loss: 0.1012 - val_accuracy: 0.9757
Epoch 11/20
237/237 [==============================] - 23s 96ms/step - loss: 0.0723 - accuracy: 0.9860 - val_loss: 0.0947 - val_accuracy: 0.9751
Epoch 12/20
237/237 [==============================] - 23s 96ms/step - loss: 0.0650 - accuracy: 0.9871 - val_loss: 0.0906 - val_accuracy: 0.9799
Epoch 13/20
237/237 [==============================] - 23s 96ms/step - loss: 0.0589 - accuracy: 0.9892 - val_loss: 0.0845 - val_accuracy: 0.9788
Epoch 14/20
237/237 [==============================] - 23s 96ms/step - loss: 0.0535 - accuracy: 0.9900 - val_loss: 0.0813 - val_accuracy: 0.9799
Epoch 15/20
237/237 [==============================] - 23s 97ms/step - loss: 0.0486 - accuracy: 0.9914 - val_loss: 0.0760 - val_accuracy: 0.9794
Epoch 16/20
237/237 [==============================] - 23s 96ms/step - loss: 0.0446 - accuracy: 0.9926 - val_loss: 0.0737 - val_accuracy: 0.9794
Epoch 17/20
237/237 [==============================] - 23s 96ms/step - loss: 0.0408 - accuracy: 0.9937 - val_loss: 0.0706 - val_accuracy: 0.9794
Epoch 18/20
237/237 [==============================] - 23s 97ms/step - loss: 0.0375 - accuracy: 0.9937 - val_loss: 0.0688 - val_accuracy: 0.9804
Epoch 19/20
237/237 [==============================] - 23s 96ms/step - loss: 0.0344 - accuracy: 0.9947 - val_loss: 0.0660 - val_accuracy: 0.9804
Epoch 20/20
237/237 [==============================] - 23s 96ms/step - loss: 0.0316 - accuracy: 0.9955 - val_loss: 0.0642 - val_accuracy: 0.9820

Evaluation of transfer learning with fixed parameters¶

Graphical evaluation¶

In [15]:
# Graphical evaluation of training performance 
acc = history_non_trainable.history['accuracy']
val_acc = history_non_trainable.history['val_accuracy']

loss = history_non_trainable.history['loss']
val_loss = history_non_trainable.history['val_loss']

epochs_range = range(epochs)

plt.figure(figsize=(11, 8))
plt.subplots_adjust(hspace = .3)
plt.subplot(2, 1, 1)
plt.plot(epochs_range, acc, label = 'Training Accuracy', color = "orange")
plt.plot(epochs_range, val_acc, label = 'Validation Accuracy', color = "blue")
plt.legend(loc = 'best')
plt.xlabel('Epochs')
plt.title('Training and Validation Accuracy', size = 13)

plt.subplot(2, 1, 2)
plt.plot(epochs_range, loss, label = 'Training Loss', color = "orange")
plt.plot(epochs_range, val_loss, label = 'Validation Loss', color = "blue")
plt.legend(loc = 'best')
plt.title('Training and Validation Loss', size = 13)
plt.xlabel('Epochs')

plt.suptitle("Transfer learning (non-trainable)", size=15)
plt.show()

Evaluating model's performance on the test dataset¶

In [16]:
# Test loss and accuracy measurments 
test_loss, test_acc = model_non_trainable.evaluate(test_batches)
print('Test loss:', test_loss)
print('Test accuracy:', test_acc)

123/123 [==============================] - 10s 77ms/step - loss: 0.0574 - accuracy: 0.9839
Test loss: 0.057424888014793396
Test accuracy: 0.9839491248130798

Evaluating the classification performance¶

i) via confussion matrix¶

In [17]:
# getting prediction labales by running the softmax results in argmax
test_labels = test_batches.classes
y_pred = model_non_trainable.predict(test_batches)
predicted_lables = np.argmax(y_pred, axis = 1)
cm =  confusion_matrix(test_labels, predicted_lables)

  1/123 [..............................] - ETA: 38s
2023-03-17 12:10:44.685157: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.

123/123 [==============================] - 9s 74ms/step
In [18]:
# dataframe containing the confussion matrix
cfm = pd.DataFrame(cm, index = class_names, columns = class_names)
In [19]:
# plotting the conffusion matrix
sns.heatmap(cfm, annot=True, fmt='d', cmap='Purples')
plt.xlabel('Predicted Label')
plt.ylabel('True Label')
plt.xticks(rotation=78)
plt.title("Trasnfer Learning (non-trainable)", size = 15)
plt.show()

ii) via designated classification performance evaluators¶

In [20]:
print("Preicision score:", precision_score(test_labels, predicted_lables, average="weighted"))
print("Recall score:", recall_score(test_labels, predicted_lables, average = "weighted"))
print("F1_score:", f1_score(test_labels, predicted_lables, average = "weighted"))

Preicision score: 0.983987378796772
Recall score: 0.9839490445859873
F1_score: 0.9839410264981292

Testing the model's prediction onto actual images¶

In [21]:
# importing the test datest again, so that this time images can be shuffled
# so that displayed images are not ordered in the same way as in the dataset 
# and variety of classes can be seen
test_data_shuffled = tf.keras.utils.image_dataset_from_directory(test_dir, shuffle = True, seed = 247)

Found 3925 files belonging to 10 classes.
In [22]:
def right_format_image(pic):
    '''
    This function returns a 
    reshaped image into 224x224 
    format in terms of height and 
    width.
    Further it normalizes the 
    pixel values within the range
    of [0, 1].
    '''
    img_size = (224, 224)
    image = tf.image.resize(pic, img_size)
    image_expanded = np.expand_dims(image, axis=0)
    image_copy = np.copy(image_expanded)
    normalized = image_copy/255.
    return normalized
In [23]:
def data_iterator(data):
    '''
    This function returns as arrays the 
    components of a batch.
    '''
    iterator = data.as_numpy_iterator()
    batch = iterator.next()
    return batch
In [24]:
# plotting images from the test dataset, with their actual and predicted from the model labels 
predicted_batch = data_iterator(test_data_shuffled)

plt.figure(figsize=(12, 12))
plt.subplots_adjust(hspace = .1, wspace=.3)
plt.suptitle("Trasnfer Learning (non-trainable)", size = 15)
for i in range(9):
    image, label = predicted_batch[0][i], predicted_batch[1][i]
    predictions = model_non_trainable.predict(right_format_image(image))
    prediction_label = class_names[predictions.argmax()]
    ax = plt.subplot(3, 3, i + 1)
    plt.imshow(image.astype(np.uint8))
    plt.title("Actual label:{};\nPredicted label:{}".format(class_names[label],
                                                           class_names[predictions.argmax()]), size = 9)
    plt.axis("off")

2023-03-17 12:10:54.625891: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.

1/1 [==============================] - 0s 345ms/step
1/1 [==============================] - 0s 15ms/step
1/1 [==============================] - 0s 15ms/step
1/1 [==============================] - 0s 15ms/step
1/1 [==============================] - 0s 16ms/step
1/1 [==============================] - 0s 14ms/step
1/1 [==============================] - 0s 15ms/step
1/1 [==============================] - 0s 14ms/step
1/1 [==============================] - 0s 15ms/step

Transfer Learning, with adaptable (learnable) parameters¶

In [25]:
# reintroduce MobileNet
mobile = tf.keras.applications.mobilenet.MobileNet()
In [26]:
# replace last 5 layers with our own 2 dense layers
x_remove_layers = mobile.layers[-5].output
output_trainable = Dense(units = 10, activation = "softmax")(x_remove_layers)
output_trainable = tf.keras.layers.Reshape(target_shape=(10,))(output_trainable)
In [27]:
# create module, that takes MobileNet and transforms it with our desired final layers,
# in place of the MobileNet's last 5 layers
model_trainable = Model(inputs = mobile.input, outputs = output_trainable)
In [28]:
# unfreeze the last 24 of the just created above module, so that they are trainable, 
# and their weights and biases will be updated (but only for those 24 final layers)
for layer in model_trainable.layers[:-24]:
    layer.trainable = False
In [29]:
model_trainable.compile(optimizer = Adam(learning_rate = 0.0001), loss = "categorical_crossentropy", metrics = ["accuracy"])
In [30]:
# when 24 layers are made trainable, it can be noticed that there
# are much more trainable parameters, roughly 2 million in this case
model_trainable.summary()

Model: "model_1"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_2 (InputLayer)        [(None, 224, 224, 3)]     0         
                                                                 
 conv1 (Conv2D)              (None, 112, 112, 32)      864       
                                                                 
 conv1_bn (BatchNormalizatio  (None, 112, 112, 32)     128       
 n)                                                              
                                                                 
 conv1_relu (ReLU)           (None, 112, 112, 32)      0         
                                                                 
 conv_dw_1 (DepthwiseConv2D)  (None, 112, 112, 32)     288       
                                                                 
 conv_dw_1_bn (BatchNormaliz  (None, 112, 112, 32)     128       
 ation)                                                          
                                                                 
 conv_dw_1_relu (ReLU)       (None, 112, 112, 32)      0         
                                                                 
 conv_pw_1 (Conv2D)          (None, 112, 112, 64)      2048      
                                                                 
 conv_pw_1_bn (BatchNormaliz  (None, 112, 112, 64)     256       
 ation)                                                          
                                                                 
 conv_pw_1_relu (ReLU)       (None, 112, 112, 64)      0         
                                                                 
 conv_pad_2 (ZeroPadding2D)  (None, 113, 113, 64)      0         
                                                                 
 conv_dw_2 (DepthwiseConv2D)  (None, 56, 56, 64)       576       
                                                                 
 conv_dw_2_bn (BatchNormaliz  (None, 56, 56, 64)       256       
 ation)                                                          
                                                                 
 conv_dw_2_relu (ReLU)       (None, 56, 56, 64)        0         
                                                                 
 conv_pw_2 (Conv2D)          (None, 56, 56, 128)       8192      
                                                                 
 conv_pw_2_bn (BatchNormaliz  (None, 56, 56, 128)      512       
 ation)                                                          
                                                                 
 conv_pw_2_relu (ReLU)       (None, 56, 56, 128)       0         
                                                                 
 conv_dw_3 (DepthwiseConv2D)  (None, 56, 56, 128)      1152      
                                                                 
 conv_dw_3_bn (BatchNormaliz  (None, 56, 56, 128)      512       
 ation)                                                          
                                                                 
 conv_dw_3_relu (ReLU)       (None, 56, 56, 128)       0         
                                                                 
 conv_pw_3 (Conv2D)          (None, 56, 56, 128)       16384     
                                                                 
 conv_pw_3_bn (BatchNormaliz  (None, 56, 56, 128)      512       
 ation)                                                          
                                                                 
 conv_pw_3_relu (ReLU)       (None, 56, 56, 128)       0         
                                                                 
 conv_pad_4 (ZeroPadding2D)  (None, 57, 57, 128)       0         
                                                                 
 conv_dw_4 (DepthwiseConv2D)  (None, 28, 28, 128)      1152      
                                                                 
 conv_dw_4_bn (BatchNormaliz  (None, 28, 28, 128)      512       
 ation)                                                          
                                                                 
 conv_dw_4_relu (ReLU)       (None, 28, 28, 128)       0         
                                                                 
 conv_pw_4 (Conv2D)          (None, 28, 28, 256)       32768     
                                                                 
 conv_pw_4_bn (BatchNormaliz  (None, 28, 28, 256)      1024      
 ation)                                                          
                                                                 
 conv_pw_4_relu (ReLU)       (None, 28, 28, 256)       0         
                                                                 
 conv_dw_5 (DepthwiseConv2D)  (None, 28, 28, 256)      2304      
                                                                 
 conv_dw_5_bn (BatchNormaliz  (None, 28, 28, 256)      1024      
 ation)                                                          
                                                                 
 conv_dw_5_relu (ReLU)       (None, 28, 28, 256)       0         
                                                                 
 conv_pw_5 (Conv2D)          (None, 28, 28, 256)       65536     
                                                                 
 conv_pw_5_bn (BatchNormaliz  (None, 28, 28, 256)      1024      
 ation)                                                          
                                                                 
 conv_pw_5_relu (ReLU)       (None, 28, 28, 256)       0         
                                                                 
 conv_pad_6 (ZeroPadding2D)  (None, 29, 29, 256)       0         
                                                                 
 conv_dw_6 (DepthwiseConv2D)  (None, 14, 14, 256)      2304      
                                                                 
 conv_dw_6_bn (BatchNormaliz  (None, 14, 14, 256)      1024      
 ation)                                                          
                                                                 
 conv_dw_6_relu (ReLU)       (None, 14, 14, 256)       0         
                                                                 
 conv_pw_6 (Conv2D)          (None, 14, 14, 512)       131072    
                                                                 
 conv_pw_6_bn (BatchNormaliz  (None, 14, 14, 512)      2048      
 ation)                                                          
                                                                 
 conv_pw_6_relu (ReLU)       (None, 14, 14, 512)       0         
                                                                 
 conv_dw_7 (DepthwiseConv2D)  (None, 14, 14, 512)      4608      
                                                                 
 conv_dw_7_bn (BatchNormaliz  (None, 14, 14, 512)      2048      
 ation)                                                          
                                                                 
 conv_dw_7_relu (ReLU)       (None, 14, 14, 512)       0         
                                                                 
 conv_pw_7 (Conv2D)          (None, 14, 14, 512)       262144    
                                                                 
 conv_pw_7_bn (BatchNormaliz  (None, 14, 14, 512)      2048      
 ation)                                                          
                                                                 
 conv_pw_7_relu (ReLU)       (None, 14, 14, 512)       0         
                                                                 
 conv_dw_8 (DepthwiseConv2D)  (None, 14, 14, 512)      4608      
                                                                 
 conv_dw_8_bn (BatchNormaliz  (None, 14, 14, 512)      2048      
 ation)                                                          
                                                                 
 conv_dw_8_relu (ReLU)       (None, 14, 14, 512)       0         
                                                                 
 conv_pw_8 (Conv2D)          (None, 14, 14, 512)       262144    
                                                                 
 conv_pw_8_bn (BatchNormaliz  (None, 14, 14, 512)      2048      
 ation)                                                          
                                                                 
 conv_pw_8_relu (ReLU)       (None, 14, 14, 512)       0         
                                                                 
 conv_dw_9 (DepthwiseConv2D)  (None, 14, 14, 512)      4608      
                                                                 
 conv_dw_9_bn (BatchNormaliz  (None, 14, 14, 512)      2048      
 ation)                                                          
                                                                 
 conv_dw_9_relu (ReLU)       (None, 14, 14, 512)       0         
                                                                 
 conv_pw_9 (Conv2D)          (None, 14, 14, 512)       262144    
                                                                 
 conv_pw_9_bn (BatchNormaliz  (None, 14, 14, 512)      2048      
 ation)                                                          
                                                                 
 conv_pw_9_relu (ReLU)       (None, 14, 14, 512)       0         
                                                                 
 conv_dw_10 (DepthwiseConv2D  (None, 14, 14, 512)      4608      
 )                                                               
                                                                 
 conv_dw_10_bn (BatchNormali  (None, 14, 14, 512)      2048      
 zation)                                                         
                                                                 
 conv_dw_10_relu (ReLU)      (None, 14, 14, 512)       0         
                                                                 
 conv_pw_10 (Conv2D)         (None, 14, 14, 512)       262144    
                                                                 
 conv_pw_10_bn (BatchNormali  (None, 14, 14, 512)      2048      
 zation)                                                         
                                                                 
 conv_pw_10_relu (ReLU)      (None, 14, 14, 512)       0         
                                                                 
 conv_dw_11 (DepthwiseConv2D  (None, 14, 14, 512)      4608      
 )                                                               
                                                                 
 conv_dw_11_bn (BatchNormali  (None, 14, 14, 512)      2048      
 zation)                                                         
                                                                 
 conv_dw_11_relu (ReLU)      (None, 14, 14, 512)       0         
                                                                 
 conv_pw_11 (Conv2D)         (None, 14, 14, 512)       262144    
                                                                 
 conv_pw_11_bn (BatchNormali  (None, 14, 14, 512)      2048      
 zation)                                                         
                                                                 
 conv_pw_11_relu (ReLU)      (None, 14, 14, 512)       0         
                                                                 
 conv_pad_12 (ZeroPadding2D)  (None, 15, 15, 512)      0         
                                                                 
 conv_dw_12 (DepthwiseConv2D  (None, 7, 7, 512)        4608      
 )                                                               
                                                                 
 conv_dw_12_bn (BatchNormali  (None, 7, 7, 512)        2048      
 zation)                                                         
                                                                 
 conv_dw_12_relu (ReLU)      (None, 7, 7, 512)         0         
                                                                 
 conv_pw_12 (Conv2D)         (None, 7, 7, 1024)        524288    
                                                                 
 conv_pw_12_bn (BatchNormali  (None, 7, 7, 1024)       4096      
 zation)                                                         
                                                                 
 conv_pw_12_relu (ReLU)      (None, 7, 7, 1024)        0         
                                                                 
 conv_dw_13 (DepthwiseConv2D  (None, 7, 7, 1024)       9216      
 )                                                               
                                                                 
 conv_dw_13_bn (BatchNormali  (None, 7, 7, 1024)       4096      
 zation)                                                         
                                                                 
 conv_dw_13_relu (ReLU)      (None, 7, 7, 1024)        0         
                                                                 
 conv_pw_13 (Conv2D)         (None, 7, 7, 1024)        1048576   
                                                                 
 conv_pw_13_bn (BatchNormali  (None, 7, 7, 1024)       4096      
 zation)                                                         
                                                                 
 conv_pw_13_relu (ReLU)      (None, 7, 7, 1024)        0         
                                                                 
 global_average_pooling2d_1   (None, 1, 1, 1024)       0         
 (GlobalAveragePooling2D)                                        
                                                                 
 dense_1 (Dense)             (None, 1, 1, 10)          10250     
                                                                 
 reshape_1 (Reshape)         (None, 10)                0         
                                                                 
=================================================================
Total params: 3,239,114
Trainable params: 1,873,930
Non-trainable params: 1,365,184
_________________________________________________________________
In [31]:
history_trainable = model_trainable.fit(x = train_batches, 
                    validation_data = validation_batches, 
                    epochs = 20)

Epoch 1/20

2023-03-17 12:10:56.817475: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.

237/237 [==============================] - ETA: 0s - loss: 0.3222 - accuracy: 0.9050
2023-03-17 12:11:20.873431: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.

237/237 [==============================] - 29s 119ms/step - loss: 0.3222 - accuracy: 0.9050 - val_loss: 0.1570 - val_accuracy: 0.9535
Epoch 2/20
237/237 [==============================] - 28s 117ms/step - loss: 0.0553 - accuracy: 0.9877 - val_loss: 0.1078 - val_accuracy: 0.9693
Epoch 3/20
237/237 [==============================] - 28s 118ms/step - loss: 0.0248 - accuracy: 0.9966 - val_loss: 0.0789 - val_accuracy: 0.9778
Epoch 4/20
237/237 [==============================] - 28s 117ms/step - loss: 0.0121 - accuracy: 0.9992 - val_loss: 0.0844 - val_accuracy: 0.9736
Epoch 5/20
237/237 [==============================] - 28s 118ms/step - loss: 0.0079 - accuracy: 0.9999 - val_loss: 0.0796 - val_accuracy: 0.9778
Epoch 6/20
237/237 [==============================] - 28s 117ms/step - loss: 0.0060 - accuracy: 0.9997 - val_loss: 0.0773 - val_accuracy: 0.9778
Epoch 7/20
237/237 [==============================] - 28s 117ms/step - loss: 0.0044 - accuracy: 0.9999 - val_loss: 0.0768 - val_accuracy: 0.9783
Epoch 8/20
237/237 [==============================] - 28s 117ms/step - loss: 0.0032 - accuracy: 1.0000 - val_loss: 0.0736 - val_accuracy: 0.9783
Epoch 9/20
237/237 [==============================] - 28s 119ms/step - loss: 0.0027 - accuracy: 1.0000 - val_loss: 0.0697 - val_accuracy: 0.9810
Epoch 10/20
237/237 [==============================] - 28s 118ms/step - loss: 0.0021 - accuracy: 1.0000 - val_loss: 0.0732 - val_accuracy: 0.9804
Epoch 11/20
237/237 [==============================] - 28s 118ms/step - loss: 0.0017 - accuracy: 1.0000 - val_loss: 0.0754 - val_accuracy: 0.9820
Epoch 12/20
237/237 [==============================] - 28s 118ms/step - loss: 0.0017 - accuracy: 1.0000 - val_loss: 0.0714 - val_accuracy: 0.9815
Epoch 13/20
237/237 [==============================] - 28s 118ms/step - loss: 0.0017 - accuracy: 1.0000 - val_loss: 0.0803 - val_accuracy: 0.9794
Epoch 14/20
237/237 [==============================] - 28s 118ms/step - loss: 0.0012 - accuracy: 0.9999 - val_loss: 0.0810 - val_accuracy: 0.9783
Epoch 15/20
237/237 [==============================] - 28s 118ms/step - loss: 0.0010 - accuracy: 1.0000 - val_loss: 0.0737 - val_accuracy: 0.9810
Epoch 16/20
237/237 [==============================] - 28s 118ms/step - loss: 9.2186e-04 - accuracy: 1.0000 - val_loss: 0.0771 - val_accuracy: 0.9794
Epoch 17/20
237/237 [==============================] - 28s 118ms/step - loss: 7.5540e-04 - accuracy: 1.0000 - val_loss: 0.0901 - val_accuracy: 0.9767
Epoch 18/20
237/237 [==============================] - 28s 118ms/step - loss: 8.0109e-04 - accuracy: 1.0000 - val_loss: 0.0923 - val_accuracy: 0.9767
Epoch 19/20
237/237 [==============================] - 28s 118ms/step - loss: 9.6637e-04 - accuracy: 0.9999 - val_loss: 0.0929 - val_accuracy: 0.9746
Epoch 20/20
237/237 [==============================] - 28s 118ms/step - loss: 8.2548e-04 - accuracy: 1.0000 - val_loss: 0.0823 - val_accuracy: 0.9757

Evaluation of transfer learning with fixed parameters¶

Graphical evaluation¶

In [32]:
# Graphical evaluation of training performance 
acc = history_trainable.history['accuracy']
val_acc = history_trainable.history['val_accuracy']

loss = history_trainable.history['loss']
val_loss = history_trainable.history['val_loss']

epochs_range = range(epochs)

plt.figure(figsize=(11, 8))
plt.subplots_adjust(hspace = .3)
plt.subplot(2, 1, 1)
plt.plot(epochs_range, acc, label = 'Training Accuracy', color = "orange")
plt.plot(epochs_range, val_acc, label = 'Validation Accuracy', color = "blue")
plt.legend(loc = 'best')
plt.xlabel('Epochs')
plt.title('Training and Validation Accuracy', size = 13)

plt.subplot(2, 1, 2)
plt.plot(epochs_range, loss, label = 'Training Loss', color = "orange")
plt.plot(epochs_range, val_loss, label = 'Validation Loss', color = "blue")
plt.legend(loc = 'best')
plt.title('Training and Validation Loss', size = 13)
plt.xlabel('Epochs')

plt.suptitle("Transfer learning (trainable)", size=15)
plt.show()

Evaluating model's performance on the test dataset¶

In [33]:
# Test loss and accuracy measurments 
test_loss, test_acc = model_trainable.evaluate(test_batches)
print('Test loss:', test_loss)
print('Test accuracy:', test_acc)

123/123 [==============================] - 9s 77ms/step - loss: 0.0717 - accuracy: 0.9786
Test loss: 0.07172290980815887
Test accuracy: 0.9785987734794617

Evaluating the classification performance¶

i) via confussion matrix¶

In [34]:
# getting prediction labales by running the softmax results in argmax
test_labels_trainable = test_batches.classes
y_pred_trainable = model_trainable.predict(test_batches)
predicted_lables_trainable = np.argmax(y_pred_trainable, axis = 1)
cm_trainable =  confusion_matrix(test_labels_trainable, predicted_lables_trainable)

2023-03-17 12:20:27.342018: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.

123/123 [==============================] - 9s 74ms/step
In [35]:
# dataframe containing the confussion matrix
cfm_trainable = pd.DataFrame(cm_trainable, index = class_names, columns = class_names)
In [36]:
# plotting the conffusion matrix
sns.heatmap(cfm_trainable, annot=True, fmt='d', cmap='Purples')
plt.xlabel('Predicted Label')
plt.ylabel('True Label')
plt.xticks(rotation=78)
plt.title("Trasnfer Learning (trainable)", size = 15)
plt.show()

ii) via designated classification performance evaluators¶

In [37]:
print("Preicision score:", precision_score(test_labels_trainable, predicted_lables_trainable, average="weighted"))
print("Recall score:", recall_score(test_labels_trainable, predicted_lables_trainable, average="weighted"))
print("F1_score:", f1_score(test_labels_trainable, predicted_lables_trainable, average="weighted"))

Preicision score: 0.9787801523100653
Recall score: 0.9785987261146497
F1_score: 0.9785276559550272

Testing the upadeted model's prediction onto actual images¶

In [38]:
# plotting images from the test dataset, with their actual and predicted from the model labels 
predicted_batch = data_iterator(test_data_shuffled)

plt.figure(figsize=(12, 12))
plt.subplots_adjust(hspace = .1, wspace=.3)
plt.suptitle("Trasnfer Learning (trainable)", size = 15)
for i in range(9):
    image, label = predicted_batch[0][i], predicted_batch[1][i]
    predictions = model_trainable.predict(right_format_image(image))
    prediction_label = class_names[predictions.argmax()]
    ax = plt.subplot(3, 3, i + 1)
    plt.imshow(image.astype(np.uint8))
    plt.title("Actual label:{};\nPredicted label:{}".format(class_names[label],
                                                           class_names[predictions.argmax()]), size = 9)
    plt.axis("off")

1/1 [==============================] - 0s 246ms/step

2023-03-17 12:20:37.127066: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.

1/1 [==============================] - 0s 18ms/step
1/1 [==============================] - 0s 22ms/step
1/1 [==============================] - 0s 15ms/step
1/1 [==============================] - 0s 16ms/step
1/1 [==============================] - 0s 14ms/step
1/1 [==============================] - 0s 17ms/step
1/1 [==============================] - 0s 15ms/step
1/1 [==============================] - 0s 14ms/step
In [ ]: